Setup Spyder for Spark

Setup Spyder for Spark

Make sure you give execute privileges Spark with Python

Setup Spyder IDE for Spark

Execute this script once when Spyder is started on Windows

# -*- coding: utf-8 -*-
import os
import sys

os.chdir("/home/cloudops/spark")
os.curdir

# Configure the environment. Set this up to the directory where
# Spark is installed
if 'SPARK_HOME' not in os.environ:
    os.environ['SPARK_HOME'] = '/opt/spark'

# Create a variable for our root path
SPARK_HOME = os.environ['SPARK_HOME']

# Add the following paths to the system path.
# Please check your installation to make sure that these zip files actually exist.
# The names might change as versions change.
sys.path.insert(0,os.path.join(SPARK_HOME,"python"))
sys.path.insert(0,os.path.join(SPARK_HOME,"python","lib"))
sys.path.insert(0,os.path.join(SPARK_HOME,"python","lib","pyspark.zip"))
sys.path.insert(0,os.path.join(SPARK_HOME,"python","lib","py4j-0.10.7-src.zip"))

# Initiate Spark context. Once this is done all other applications can run
from pyspark import SparkContext
from pyspark import SparkConf

# Optionally configure Spark Settings
conf = SparkConf()
conf.set("spark.executor.memory", "1g")
conf.set("spark.cores.max", "2")

conf.setAppName("Spark-Test")

# Initialize SparkContext. Run only once!!!
# Otherwise you get multiple Context Error.
sc = SparkContext('local', conf=conf)

# Test to make sure everything works.
lines = sc.textFile("data/auto-data.csv")
lines.count()